% %load('tokens.mat')
% fid = fopen('dictionary.txt','rt');
% numTokens = 0;
% while (fgets(fid) ~= -1),
%     numTokens = numTokens+1;
% end
% numTokens=8;
% %numTokens=length(tokens);
% %down_counter=0;
% % fileoffset=10;
% % filestr='translateddata/outputfile2009-06-';
% % datestr='2009-06-10';
% % findindex=0;
% % for i=1:length(date)
% %     if isequal(char(date(i)),datestr)
% %         findindex=i
% %     end
% % end
% 
% path='hourlytweets1';
% foldercontent=dir(path);
% numDays=size(foldercontent,1)-3;
% datestr_start=1;
% datestr_length=9;
% 
% x=zeros(numTokens,numDays*24);
% 
% stocks_status=[];
% tweetdate=[];
for i=141:numDays
    filename=char(foldercontent(i+3).name);
    datestr=filename(datestr_start:datestr_start+datestr_length);
    tweetdate=[tweetdate;datestr];

    for dayindex=0:23
        fullfilename=[path '/' char(foldercontent(i+3).name) '/' sprintf('%02d',dayindex) '.txt']
        fid = fopen(fullfilename,'rt');
        numTweets = 0;
        while (fgets(fid) ~= -1),
            numTweets = numTweets+1;
        end
        fclose(fid);
        trainMatrix = textread(fullfilename,'%d', 50*numTweets);
        trainMatrix=trainMatrix(trainMatrix>0);
        trainMatrix=trainMatrix(trainMatrix<=numTokens);
        for j=1:length(trainMatrix)
            x(trainMatrix(j),(i-2)*24+dayindex+1)=x(trainMatrix(j),(i-2)*24+dayindex+1)+1;
        end
    end    
end
perhourtweets=x;
save('perhourttweet.mat','perhourtweets','tweetdate')
